package com.alpha;

import org.apache.commons.httpclient.*;
import org.apache.commons.httpclient.methods.*;

import org.apache.commons.httpclient.HttpMethod;
import java.io.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Vector;
import java.util.regex.Pattern;

public class FDAAnalyzer {
	public static final String FAILD_URLS_FILE = "c:\\alphapharm\\files\\_failedURLs.txt";
	public static final String  EGYGENERIC_EGYBOXMAPPIN = "c:\\alphapharm\\files\\meta\\egygeneric_egyboxMapping.csv";
	public static final String  FDA_EGYMAPPING = "c:\\alphapharm\\files\\meta\\FDA_EgyptianMapping.txt";
	public static final String  FDA_DRUGS_FILE= "c:\\alphapharm\\files\\_drugs@fda.csv";
	public static final String  REMAINING_LINKS= "c:\\alphapharm\\files\\remaininglinks.obj";

	
	public static final String  SEARCH_URL = "http://www.accessdata.fda.gov/scripts/cder/drugsatfda/index.cfm?fuseaction=Search.SearchResults_Browse&DrugInitial=";
	public static final String  PREFIX= "http://www.accessdata.fda.gov/scripts/cder/drugsatfda/";
	private static Map<String, Integer> egyptianBoxes;
	private static Set<String> egyptianBoxesCompnents = new HashSet<String>();
	public static Set<String> fdaBoxesCompnents = new HashSet<String>();
	public static Map<String, String> fda_egyMapping = new HashMap<String, String>();
	public static Map<String, String> egygeneric_egyboxMapping = new HashMap<String, String>();

	public static int i = 0;

	public static void main(String[] args) throws HttpException, IOException, ClassNotFoundException {
		egygeneric_egyboxMapping = loadegygeneric_egyboxMapping();
		egyptianBoxes = loadEgyptianBoxes();
		fda_egyMapping = loadMapping();

		ceateCsv();
		ceatefailedURLsFile();
		// String searchUrl =
		// "http://www.accessdata.fda.gov/scripts/cder/drugsatfda/index.cfm?fuseaction=Search.SearchResults_Browse&DrugInitial=D";
		// String drugurl =
		// "http://www.accessdata.fda.gov/scripts/cder/drugsatfda/index.cfm?fuseaction=Search.Overview&DrugName=EPOPROSTENOL%20SODIUM";
		// getFDADrugsFromDrugURL(searchUrl, drugurl);

		//GetAllFDADrugsLinks();
		 GetAllFDADrugsFromLocalStore();
		//getRemainingDrugs();
		storeFDABoxes(fdaBoxesCompnents);
	}

	private static void getRemainingDrugs() throws IOException, ClassNotFoundException {
		FileInputStream fis = new FileInputStream(REMAINING_LINKS);
		ObjectInputStream ois = new ObjectInputStream(fis);
		ArrayList<String[]> links = (ArrayList<String[]>) ois.readObject();
		ois.close();
		fis.close();
		getDrugsFromLinks(links);
		
		
	}

	private static void getDrugsFromLinks(ArrayList<String[]> links) throws HttpException, IOException {
		ArrayList<String[]> remaininglinks= (ArrayList<String[]>) links.clone();
		for (String[] i : links) {
			// String[] i = links.get(0);
			// System.out.println(i[0] +": "+i[1]);
			getFDADrugsFromDrugURL(i[2],  PREFIX +i[0]);
			remaininglinks.remove(i);
			saveRemainingLinks(remaininglinks);
			//callurl3(client, prefix + i[0],  new   ArrayList<String>());
		}//		
	}

	private static Map<String, String> loadegygeneric_egyboxMapping()
			throws IOException {
		HashMap<String, String> egygeneric_egyboxMapping = new HashMap<String, String>();

		FileInputStream fstream = new FileInputStream(
				EGYGENERIC_EGYBOXMAPPIN);
		// Get the object of DataInputStream
		DataInputStream in = new DataInputStream(fstream);
		BufferedReader br = new BufferedReader(new InputStreamReader(in));
		String strLine;

		while ((strLine = br.readLine()) != null) {
			String[] mapping = strLine.split(";");
			egygeneric_egyboxMapping.put(mapping[0].trim(), mapping[1].trim());

		}
		// Close the input stream
		in.close();

		System.out.println(Arrays.asList(egygeneric_egyboxMapping.keySet()));
		return egygeneric_egyboxMapping;
	}

	private static void storeFDABoxes(Set<String> fdaBoxesCompnents2)
			throws IOException {
		DataOutputStream out = new DataOutputStream(new FileOutputStream(
				"c:\\alphapharm\\files\\_FDABoxesCompnents.txt"));
		BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out));
		for (Object boxCompnent : fdaBoxesCompnents2.toArray()) {
			writer.write(boxCompnent.toString() + "\n");
		}
		writer.close();
		out.close();
	}

	private static void GetAllFDADrugsFromLocalStore() throws IOException {
		File srcdir = new File("c:\\alphapharm\\files\\source");
		File[] files = srcdir.listFiles();
		for (File file : files) {

			System.out.println(file);
			FileInputStream fstream = new FileInputStream(file);
			DataInputStream in = new DataInputStream(fstream);
			BufferedReader br = new BufferedReader(new InputStreamReader(in));
			StringBuffer srcContent = new StringBuffer();
			String strLine;
			while ((strLine = br.readLine()) != null) {
				srcContent.append(strLine);
			}
			List datalist = HTMLLinkExtrator.grabDataTable(srcContent
					.toString());
			for (Object drugdata : datalist) {

				storeCsv((Map) drugdata, "");

			}
		}
	}

	private static Map<String, String> loadMapping() throws IOException {
		HashMap<String, String> fdaegyMapping = new HashMap<String, String>();

		FileInputStream fstream = new FileInputStream( FDA_EGYMAPPING);
		// Get the object of DataInputStream
		DataInputStream in = new DataInputStream(fstream);
		BufferedReader br = new BufferedReader(new InputStreamReader(in));
		String strLine;

		while ((strLine = br.readLine()) != null) {
			String[] mapping = strLine.split(";");
			fdaegyMapping.put(mapping[0].trim(), mapping[1].trim());

		}
		// Close the input stream
		in.close();

		System.out.println(Arrays.asList(fdaegyMapping.keySet()));
		return fdaegyMapping;
	}

	private static Map<String, Integer> loadEgyptianBoxes() throws IOException {
		HashMap<String, Integer> egyBoxes = new HashMap<String, Integer>();

		FileInputStream fstream = new FileInputStream(
				"c:\\alphapharm\\files\\meta\\EgyptianDrugs.txt");
		// Get the object of DataInputStream
		DataInputStream in = new DataInputStream(fstream);
		BufferedReader br = new BufferedReader(new InputStreamReader(in));
		String strLine;

		while ((strLine = br.readLine()) != null) {

			String[] boxlist = formatBox(strLine);
			String box = (Arrays.toString(boxlist));
			Integer boxvalue = egyBoxes.get(box);
			// System.out.println(box + ": " + egyBoxes.get(box) + " : ");
			if (boxvalue != null) {
				egyBoxes.put(box, ++boxvalue);
			} else {
				egyBoxes.put(box, 1);
			}

		}
		// Close the input stream
		in.close();

		storeBoxes(egyBoxes);
		storeBoxesCompnents(egyptianBoxesCompnents);

		return egyBoxes;
	}

	private static void storeBoxesCompnents(Set<String> egyptianBoxesCompnents)
			throws IOException {
		DataOutputStream out = new DataOutputStream(new FileOutputStream(
				"c:\\alphapharm\\files\\_EgyptianBoxesCompnents.txt"));
		BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out));
		for (Object boxCompnent : egyptianBoxesCompnents.toArray()) {
			writer.write(boxCompnent.toString() + "\n");
		}
		writer.close();
		out.close();

	}

	private static void storeBoxes(HashMap<String, Integer> egyBoxes)
			throws IOException {
		DataOutputStream out = new DataOutputStream(new FileOutputStream(
				"c:\\alphapharm\\files\\_EgyptianBoxes.txt"));
		BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out));

		for (Object box : egyBoxes.keySet().toArray()) {

			writer.write(box.toString() + "|" + egyBoxes.get(box) + "\n");
		}
		writer.close();
		out.close();

	}

	private static String[] formatBox(String ingredents) {

		ingredents = ingredents.toUpperCase().trim();
		String[] boxList = ingredents.split("[ ]*\\+[ ]*");

		for (int x = 0; x < boxList.length; x++) {
			String egybox = egygeneric_egyboxMapping.get(boxList[x]);
			if (egybox != null) {
				boxList[x] = egybox.trim();
			}
		}
		egyptianBoxesCompnents.addAll(Arrays.asList(boxList));

		Arrays.sort(boxList);

		return boxList;
	}

	public static void getFDADrugsFromDrugURL(String searchURL, String drugUrl)
			throws HttpException, IOException {

		HttpClient client = new HttpClient();

		GetMethod method = new GetMethod(searchURL);

		// Execute the method.
		byte[] responseBody = new byte[1];
		try {
			int statusCode = client.executeMethod(method);
			responseBody = method.getResponseBody();
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		String x = new String(responseBody);
		callurl3(client, drugUrl,
				 new ArrayList<String>());

	}

	public static void GetAllFDADrugsLinks() throws HttpException, IOException {
		ArrayList <String []> links = new ArrayList <String []>();
		char letter;
		for (letter = 'A'; letter <= 'Z'; letter++) {
			String url =  SEARCH_URL + letter + "&&StartRow=1&StepSize=100000";
			
			links.addAll( getFDADrugsFromSearchResult(url));
		
		}
		saveRemainingLinks(links);
        

	}

	private static void saveRemainingLinks(ArrayList<String[]> links) throws IOException {
		FileOutputStream fos = new FileOutputStream(REMAINING_LINKS);
		ObjectOutputStream oos=  new ObjectOutputStream (fos);
		oos.writeObject(links);
		oos.close();
		fos.close();
		
	}

	public static Vector<String[]> getFDADrugsFromSearchResult(String searchurl)
			throws HttpException, IOException {

		HttpClient client = new HttpClient();

		GetMethod method = new GetMethod(searchurl);

		// Execute the method.
		int statusCode = client.executeMethod(method);
		byte[] responseBody = method.getResponseBody();

		String x = new String(responseBody);

		Vector<String[]> links = HTMLLinkExtrator.grabHTMLLinks(x, searchurl);
        return links;

	}

	private static void callurl3(HttpClient client, String url3,  List<String> urlreferer) throws IOException
			 {
		urlreferer.add(url3);

		System.out.println("URL:"+url3);
		System.out.println("urlreferer:"+urlreferer);
		

		GetMethod method = new GetMethod(url3);
		method.setFollowRedirects(false);

		try {
			int retruncode = client.executeMethod(method);
		} catch (Exception e) {
			storeFailedURLS(urlreferer);
            e.printStackTrace();			
return;
		}
		Header locationHeader = method.getResponseHeader("location");
		String redirectLocation = null;
		if (locationHeader != null) {
			redirectLocation = locationHeader.getValue();

			callurl3(client,
					"http://www.accessdata.fda.gov/scripts/cder/drugsatfda/"
							+ redirectLocation,  urlreferer);
		} else {

			byte[] responseBody;
			String x ="";
			try {
				//responseBody = method.getResponseBody();
				//x = new String(responseBody);
                x = method.getResponseBodyAsString();
			} catch (Exception e) {
				storeFailedURLS(urlreferer);
				e.printStackTrace();
			}
			if (url3.endsWith("DrugDetails")) {
				saveSourceFile(x);
				List datalist = HTMLLinkExtrator.grabDataTable(x);
				for (Object drugdata : datalist) {
					storeCsv((Map) drugdata, url3);

				}

			}

			else {
				String prefix = "http://www.accessdata.fda.gov/scripts/cder/drugsatfda/";
				Vector<String[]> links = HTMLLinkExtrator
						.grabHTMLDrugDetailsLinks(x);
				for (String[] i : links) {

					callurl3(client, prefix + i[0],  urlreferer);
				}

			}

			method.releaseConnection();

		}

	}

	private static void storeFailedURLS(List<String> urlreferer) throws IOException {
		FileWriter outFile = new FileWriter(
				FAILD_URLS_FILE, true);
		PrintWriter out = new PrintWriter(outFile);
 		out.print("\n"+urlreferer.get(0));
		 
		out.close();
		outFile.close();
		
	}

	private static void saveSourceFile(String x) throws IOException {
		String file = "c:\\alphapharm\\files\\source\\src_"
				+ (System.currentTimeMillis()) + ".htm";
		FileWriter outFile = new FileWriter(file, true);
		PrintWriter out = new PrintWriter(outFile);
		out.print("\n" + (x));

		out.close();
		outFile.close();

	}

	private static void storeCsv(Map drugdata, String url3) throws IOException {
		FileWriter outFile = new FileWriter(	FDA_DRUGS_FILE, true);
		PrintWriter out = new PrintWriter(outFile);
		System.out.println("Saving: " + drugdata.get("Drug Name(s)"));
		out.print("\n" + drugdata.get("Drug Name(s)"));
		out.print("|" + drugdata.get("FDA Application No."));
		out.print("|" + drugdata.get("Review Classification"));
		out.print("|" + drugdata.get("Company"));
		out.print("|"
				+ drugdata.get("Original Approval or Tentative Approval Date"));
		out.print("|" + drugdata.get("Original Approval Year"));

		out.print("|" + drugdata.get("Dosage Form/Route"));
		out.print("|" + drugdata.get("Strength"));
		out.print("|" + drugdata.get("Chemical Type"));

		out.print("|" + drugdata.get("MarketingStatus"));
		out.print("|" + drugdata.get("RLD"));
		out.print("|" + ((String) drugdata.get("Box")));
		out.print("|"
				+ ((String) drugdata.get("Active Ingredient(s)")).replaceAll(
						"&nbsp;", ""));
		out.print("|" + (egyptianBoxes.get((String) drugdata.get("Box"))));

		// System.out.println(drugdata.get("Active Ingredient(s)").toString().replaceAll("&nbsp;",
		// ""));

		// System.out.println("get get(drugdata.get(Active Ingredient(s))"
		// + egyptianBoxes.get(drugdata.get("Active Ingredient(s)")));
		// out.print("," + url3);
		out.close();
		outFile.close();

	}

	private static void ceateCsv() throws IOException {

		FileWriter outFile = new FileWriter(FDA_DRUGS_FILE, true);
		PrintWriter out = new PrintWriter(outFile);
		System.out.println("creating File: " + FDA_DRUGS_FILE);
		out.print("\n" + ("Drug Name(s)"));
		out.print("|" + "FDA Application No.");
		out.print("|" + "Review Classification");
		out.print("|" + ("Company"));
		out.print("|" + ("Original Approval or Tentative Approval Date"));
		out.print("|" + ("Year"));

		out.print("|" + ("Dosage Form/Route"));
		out.print("|" + ("Strength"));
		out.print("|" + ("Chemical Type"));

		out.print("|" + ("MarketingStatus"));
		out.print("|" + ("RLD"));
		out.print("|" + ("Box"));
		out.print("|" + ("Active Ingredient(s)"));
		out.print("|" + ("Count In Egypt"));

		// out.print("," + url3);
		out.close();
		outFile.close();

	}
	private static void ceatefailedURLsFile() throws IOException {
		FileWriter outFile = new FileWriter(FAILD_URLS_FILE, true);
		PrintWriter out = new PrintWriter(outFile);
		System.out.println("creating File: " + FAILD_URLS_FILE);
		out.print("\n" + ("Failed URLS"));
		
        
		out.close();
		outFile.close();

	}

}